{
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.12",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [],
   "dockerImageVersionId": 30616,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook",
   "isGpuEnabled": true
  }
 },
 "nbformat_minor": 4,
 "nbformat": 4,
 "cells": [
  {
   "cell_type": "code",
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n",
    "\n",
    "seed = 42\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-13T08:01:15.341311Z",
     "iopub.execute_input": "2023-12-13T08:01:15.342146Z",
     "iopub.status.idle": "2023-12-13T08:01:19.541259Z",
     "shell.execute_reply.started": "2023-12-13T08:01:15.342110Z",
     "shell.execute_reply": "2023-12-13T08:01:19.539607Z"
    },
    "trusted": true
   },
   "execution_count": 1,
   "outputs": [
    {
     "name": "stderr",
     "text": "/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.3\n  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n",
     "output_type": "stream"
    },
    {
     "name": "stdout",
     "text": "sys.version_info(major=3, minor=10, micro=12, releaselevel='final', serial=0)\nmatplotlib 3.7.4\nnumpy 1.24.3\npandas 2.1.3\nsklearn 1.2.2\ntorch 2.0.0\ncuda:0\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 数据准备"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "!wget https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-13T08:01:19.543345Z",
     "iopub.execute_input": "2023-12-13T08:01:19.544549Z",
     "iopub.status.idle": "2023-12-13T08:01:20.745715Z",
     "shell.execute_reply.started": "2023-12-13T08:01:19.544511Z",
     "shell.execute_reply": "2023-12-13T08:01:20.744562Z"
    },
    "trusted": true
   },
   "execution_count": 2,
   "outputs": [
    {
     "name": "stdout",
     "text": "--2023-12-13 08:01:20--  https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\nResolving storage.googleapis.com (storage.googleapis.com)... 173.194.215.207, 173.194.216.207, 173.194.217.207, ...\nConnecting to storage.googleapis.com (storage.googleapis.com)|173.194.215.207|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 1115394 (1.1M) [text/plain]\nSaving to: ‘shakespeare.txt’\n\nshakespeare.txt     100%[===================>]   1.06M  --.-KB/s    in 0.01s   \n\n2023-12-13 08:01:20 (94.9 MB/s) - ‘shakespeare.txt’ saved [1115394/1115394]\n\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "# https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\n",
    "#文件已经下载好了\n",
    "with open(\"./shakespeare.txt\", \"r\", encoding=\"utf8\") as file:\n",
    "    text = file.read()\n",
    "\n",
    "print(\"length\", len(text))\n",
    "print(text[0:100])"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-13T08:01:20.747517Z",
     "iopub.execute_input": "2023-12-13T08:01:20.747963Z",
     "iopub.status.idle": "2023-12-13T08:01:20.758411Z",
     "shell.execute_reply.started": "2023-12-13T08:01:20.747923Z",
     "shell.execute_reply": "2023-12-13T08:01:20.757562Z"
    },
    "trusted": true
   },
   "execution_count": 3,
   "outputs": [
    {
     "name": "stdout",
     "text": "length 1115394\nFirst Citizen:\nBefore we proceed any further, hear me speak.\n\nAll:\nSpeak, speak.\n\nFirst Citizen:\nYou\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 构造字典"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "# 1. generate vocab\n",
    "# 2. build mapping char->id\n",
    "# 3. data -> id_data  把数据都转为id\n",
    "# 4. a b c d [EOS] -> [BOS] b c d  预测下一个字符生成的模型，也就是输入是a，输出就是b\n",
    "\n",
    "#去重，留下独立字符，并排序\n",
    "vocab = sorted(set(text))\n",
    "print(len(vocab))\n",
    "print(vocab)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-13T08:01:20.760582Z",
     "iopub.execute_input": "2023-12-13T08:01:20.760920Z",
     "iopub.status.idle": "2023-12-13T08:01:20.787130Z",
     "shell.execute_reply.started": "2023-12-13T08:01:20.760888Z",
     "shell.execute_reply": "2023-12-13T08:01:20.786223Z"
    },
    "trusted": true
   },
   "execution_count": 4,
   "outputs": [
    {
     "name": "stdout",
     "text": "65\n['\\n', ' ', '!', '$', '&', \"'\", ',', '-', '.', '3', ':', ';', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "for idx,char in enumerate(['how','are','you']):\n",
    "    print(idx,char)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-13T08:01:20.788334Z",
     "iopub.execute_input": "2023-12-13T08:01:20.788946Z",
     "iopub.status.idle": "2023-12-13T08:01:20.797490Z",
     "shell.execute_reply.started": "2023-12-13T08:01:20.788908Z",
     "shell.execute_reply": "2023-12-13T08:01:20.796664Z"
    },
    "trusted": true
   },
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "text": "0 how\n1 are\n2 you\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "#每个字符都编好号，enumerate对每一个位置编号，生成的是列表中是元组，下面字典生成式\n",
    "char2idx = {char:idx for idx, char in enumerate(vocab)}\n",
    "print(char2idx)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-13T08:01:20.798642Z",
     "iopub.execute_input": "2023-12-13T08:01:20.798958Z",
     "iopub.status.idle": "2023-12-13T08:01:20.807780Z",
     "shell.execute_reply.started": "2023-12-13T08:01:20.798924Z",
     "shell.execute_reply": "2023-12-13T08:01:20.806904Z"
    },
    "trusted": true
   },
   "execution_count": 6,
   "outputs": [
    {
     "name": "stdout",
     "text": "{'\\n': 0, ' ': 1, '!': 2, '$': 3, '&': 4, \"'\": 5, ',': 6, '-': 7, '.': 8, '3': 9, ':': 10, ';': 11, '?': 12, 'A': 13, 'B': 14, 'C': 15, 'D': 16, 'E': 17, 'F': 18, 'G': 19, 'H': 20, 'I': 21, 'J': 22, 'K': 23, 'L': 24, 'M': 25, 'N': 26, 'O': 27, 'P': 28, 'Q': 29, 'R': 30, 'S': 31, 'T': 32, 'U': 33, 'V': 34, 'W': 35, 'X': 36, 'Y': 37, 'Z': 38, 'a': 39, 'b': 40, 'c': 41, 'd': 42, 'e': 43, 'f': 44, 'g': 45, 'h': 46, 'i': 47, 'j': 48, 'k': 49, 'l': 50, 'm': 51, 'n': 52, 'o': 53, 'p': 54, 'q': 55, 'r': 56, 's': 57, 't': 58, 'u': 59, 'v': 60, 'w': 61, 'x': 62, 'y': 63, 'z': 64}\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "# 把vocab从列表变为ndarray\n",
    "idx2char = np.array(vocab)\n",
    "print(idx2char)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-13T08:01:20.808899Z",
     "iopub.execute_input": "2023-12-13T08:01:20.809170Z",
     "iopub.status.idle": "2023-12-13T08:01:20.819342Z",
     "shell.execute_reply.started": "2023-12-13T08:01:20.809147Z",
     "shell.execute_reply": "2023-12-13T08:01:20.818365Z"
    },
    "trusted": true
   },
   "execution_count": 7,
   "outputs": [
    {
     "name": "stdout",
     "text": "['\\n' ' ' '!' '$' '&' \"'\" ',' '-' '.' '3' ':' ';' '?' 'A' 'B' 'C' 'D' 'E'\n 'F' 'G' 'H' 'I' 'J' 'K' 'L' 'M' 'N' 'O' 'P' 'Q' 'R' 'S' 'T' 'U' 'V' 'W'\n 'X' 'Y' 'Z' 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o'\n 'p' 'q' 'r' 's' 't' 'u' 'v' 'w' 'x' 'y' 'z']\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "#把字符都转换为id\n",
    "text_as_int = np.array([char2idx[c] for c in text])\n",
    "print(text_as_int.shape)\n",
    "print(len(text_as_int))\n",
    "print(text_as_int[0:10])\n",
    "print(text[0:10])"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-13T08:01:20.820644Z",
     "iopub.execute_input": "2023-12-13T08:01:20.821019Z",
     "iopub.status.idle": "2023-12-13T08:01:21.031024Z",
     "shell.execute_reply.started": "2023-12-13T08:01:20.820985Z",
     "shell.execute_reply": "2023-12-13T08:01:21.030069Z"
    },
    "trusted": true
   },
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "text": "(1115394,)\n1115394\n[18 47 56 57 58  1 15 47 58 47]\nFirst Citi\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 把莎士比亚文集分成一个一个的样本"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "class CharDataset(Dataset):\n",
    "    def __init__(self, text_as_int, seq_length):\n",
    "        self.sub_len = seq_length + 1\n",
    "        self.text_as_int = text_as_int\n",
    "        self.num_seq = len(text_as_int) // self.sub_len\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        return self.text_as_int[index * self.sub_len: (index + 1) * self.sub_len]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.num_seq\n",
    "    \n",
    "def collat_fct(batch):\n",
    "    src_list = []\n",
    "    trg_list = []\n",
    "    for part in batch:\n",
    "        src_list.append(part[:-1])\n",
    "        trg_list.append(part[1:])\n",
    "        \n",
    "    src_list = np.array(src_list)\n",
    "    trg_list = np.array(trg_list)\n",
    "    return torch.Tensor(src_list).to(dtype=torch.int64), torch.Tensor(trg_list).to(dtype=torch.int64)\n",
    "        \n",
    "\n",
    "train_ds = CharDataset(text_as_int, 100)\n",
    "train_dl = DataLoader(train_ds, batch_size=64, shuffle=True, collate_fn=collat_fct)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-13T08:01:21.032444Z",
     "iopub.execute_input": "2023-12-13T08:01:21.033169Z",
     "iopub.status.idle": "2023-12-13T08:01:21.043654Z",
     "shell.execute_reply.started": "2023-12-13T08:01:21.033131Z",
     "shell.execute_reply": "2023-12-13T08:01:21.042493Z"
    },
    "trusted": true
   },
   "execution_count": 9,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 定义模型"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "class CharLSTM(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim=256, hidden_dim=1024):\n",
    "        super(CharLSTM, self).__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_dim, vocab_size)\n",
    "        \n",
    "    def forward(self, x, hidden=None):\n",
    "        x = self.embedding(x)\n",
    "        output, hidden = self.lstm(x, hidden)\n",
    "        x = self.fc(output)\n",
    "        return x, hidden\n",
    "    \n",
    "    \n",
    "vocab_size = len(vocab)\n",
    "sample_inputs = torch.randint(0, vocab_size, (2, 128))\n",
    "    \n",
    "print(\"{:=^80}\".format(\" 一层单向 LSTM \"))       \n",
    "for key, value in CharLSTM(vocab_size).named_parameters():\n",
    "    print(f\"{key:^40}paramerters num: {np.prod(value.shape)}\")\n",
    "    \n",
    "CharLSTM(vocab_size)(sample_inputs)[0].shape"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-13T08:01:21.048171Z",
     "iopub.execute_input": "2023-12-13T08:01:21.048552Z",
     "iopub.status.idle": "2023-12-13T08:01:21.466222Z",
     "shell.execute_reply.started": "2023-12-13T08:01:21.048518Z",
     "shell.execute_reply": "2023-12-13T08:01:21.465240Z"
    },
    "trusted": true
   },
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "text": "================================== 一层单向 LSTM ===================================\n            embedding.weight            paramerters num: 16640\n           lstm.weight_ih_l0            paramerters num: 1048576\n           lstm.weight_hh_l0            paramerters num: 4194304\n            lstm.bias_ih_l0             paramerters num: 4096\n            lstm.bias_hh_l0             paramerters num: 4096\n               fc.weight                paramerters num: 66560\n                fc.bias                 paramerters num: 65\n",
     "output_type": "stream"
    },
    {
     "execution_count": 10,
     "output_type": "execute_result",
     "data": {
      "text/plain": "torch.Size([2, 128, 65])"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "4 * 1024*256"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-13T02:36:48.216578Z",
     "start_time": "2023-12-13T02:36:48.192464200Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "execution": {
     "iopub.status.busy": "2023-12-13T08:01:21.467328Z",
     "iopub.execute_input": "2023-12-13T08:01:21.467643Z",
     "iopub.status.idle": "2023-12-13T08:01:21.475624Z",
     "shell.execute_reply.started": "2023-12-13T08:01:21.467617Z",
     "shell.execute_reply": "2023-12-13T08:01:21.474615Z"
    },
    "trusted": true
   },
   "execution_count": 11,
   "outputs": [
    {
     "execution_count": 11,
     "output_type": "execute_result",
     "data": {
      "text/plain": "1048576"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "4 * 1024*1024"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-13T02:37:02.308627800Z",
     "start_time": "2023-12-13T02:37:02.296635500Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "execution": {
     "iopub.status.busy": "2023-12-13T08:01:21.477001Z",
     "iopub.execute_input": "2023-12-13T08:01:21.477377Z",
     "iopub.status.idle": "2023-12-13T08:01:21.487279Z",
     "shell.execute_reply.started": "2023-12-13T08:01:21.477349Z",
     "shell.execute_reply": "2023-12-13T08:01:21.486283Z"
    },
    "trusted": true
   },
   "execution_count": 12,
   "outputs": [
    {
     "execution_count": 12,
     "output_type": "execute_result",
     "data": {
      "text/plain": "4194304"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 训练"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        \"\"\"\n",
    "        Save checkpoints each save_epoch epoch. \n",
    "        We save checkpoint by epoch in this implementation.\n",
    "        Usually, training scripts with pytorch evaluating model and save checkpoint by step.\n",
    "\n",
    "        Args:\n",
    "            save_dir (str): dir to save checkpoint\n",
    "            save_epoch (int, optional): the frequency to save checkpoint. Defaults to 1.\n",
    "            save_best_only (bool, optional): If True, only save the best model or save each model at every epoch.\n",
    "        \"\"\"\n",
    "        self.save_dir = save_dir\n",
    "        self.save_step = save_step\n",
    "        self.save_best_only = save_best_only\n",
    "        self.best_metrics = -1\n",
    "        \n",
    "        # mkdir\n",
    "        if not os.path.exists(self.save_dir):\n",
    "            os.mkdir(self.save_dir)\n",
    "        \n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return\n",
    "        \n",
    "        if self.save_best_only:\n",
    "            assert metric is not None\n",
    "            if metric >= self.best_metrics:\n",
    "                # save checkpoints\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"best.ckpt\"))\n",
    "                # update best metrics\n",
    "                self.best_metrics = metric\n",
    "        else:\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "\n"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-13T08:01:21.488471Z",
     "iopub.execute_input": "2023-12-13T08:01:21.488777Z",
     "iopub.status.idle": "2023-12-13T08:01:21.499195Z",
     "shell.execute_reply.started": "2023-12-13T08:01:21.488751Z",
     "shell.execute_reply": "2023-12-13T08:01:21.498155Z"
    },
    "trusted": true
   },
   "execution_count": 13,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "len(train_loader)"
   ],
   "metadata": {},
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# 训练\n",
    "def training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    save_ckpt_callback=None,\n",
    "    stateful=False      # 想用stateful，batch里的数据就必须连续，不能打乱\n",
    "    ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "    }\n",
    "    \n",
    "    global_step = 0\n",
    "    model.train()\n",
    "    hidden = None\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for datas, labels in train_loader:\n",
    "                datas = datas.to(device)\n",
    "                labels = labels.to(device)\n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "                # 模型前向计算\n",
    "                logits, hidden = model(datas, hidden=hidden if stateful else None)\n",
    "                # 计算损失\n",
    "                loss = loss_fct(logits.reshape(-1, vocab_size), labels.reshape(-1))\n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    " \n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "                \n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"step\": global_step\n",
    "                })\n",
    "   \n",
    "                # 保存模型权重 save model checkpoint\n",
    "                if save_ckpt_callback is not None:\n",
    "                    save_ckpt_callback(global_step, model.state_dict(), metric=-loss)\n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "        \n",
    "    return record_dict\n",
    "        \n",
    "\n",
    "epoch = 100\n",
    "\n",
    "model = CharLSTM(vocab_size=vocab_size)\n",
    "\n",
    "# 1. 定义损失函数 采用交叉熵损失 \n",
    "loss_fct = nn.CrossEntropyLoss()\n",
    "# 2. 定义优化器 采用 adam\n",
    "# Optimizers specified in the torch.optim package\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "\n",
    "# save best\n",
    "if not os.path.exists(\"checkpoints\"):\n",
    "    os.makedirs(\"checkpoints\")\n",
    "save_ckpt_callback = SaveCheckpointsCallback(\"checkpoints/text_generation_lstm\", save_step=1000, save_best_only=True)\n",
    "\n",
    "\n",
    "model = model.to(device)\n",
    "record = training(\n",
    "    model, \n",
    "    train_dl, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    "    )"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-13T08:01:21.500386Z",
     "iopub.execute_input": "2023-12-13T08:01:21.500707Z",
     "iopub.status.idle": "2023-12-13T08:14:21.838651Z",
     "shell.execute_reply.started": "2023-12-13T08:01:21.500682Z",
     "shell.execute_reply": "2023-12-13T08:14:21.837620Z"
    },
    "trusted": true
   },
   "execution_count": 14,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "  0%|          | 0/17300 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "387741bc4cbe43ba82da2ad3c188df99"
      }
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "plt.plot([i[\"step\"] for i in record[\"train\"][::50]], [i[\"loss\"] for i in record[\"train\"][::50]], label=\"train\")\n",
    "plt.grid()\n",
    "plt.show()"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-13T08:14:21.840148Z",
     "iopub.execute_input": "2023-12-13T08:14:21.840470Z",
     "iopub.status.idle": "2023-12-13T08:14:22.158455Z",
     "shell.execute_reply.started": "2023-12-13T08:14:21.840441Z",
     "shell.execute_reply": "2023-12-13T08:14:22.157399Z"
    },
    "trusted": true
   },
   "execution_count": 15,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 640x480 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAikAAAGdCAYAAADXIOPgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8WgzjOAAAACXBIWXMAAA9hAAAPYQGoP6dpAABaXElEQVR4nO3deVhTV/4G8DeBENawiOygIO47LhR3667TatuxjnVGu2jbqXbs2J/tONNpaztTnW52tzpdbO1YWzutTls31KKiuEBFxQVBEFB2FMIaQnJ+fyS5EAFbFs3FvJ/n4Qm5uTc5+XpNXs4991yFEEKAiIiISGaUtm4AERERUVMYUoiIiEiWGFKIiIhIlhhSiIiISJYYUoiIiEiWGFKIiIhIlhhSiIiISJYYUoiIiEiWHG3dgF/DaDQiNzcXHh4eUCgUtm4OERER/QpCCJSXlyMoKAhKZcv7RTpESMnNzUVoaKitm0FEREStkJOTg5CQkBZv1yFCioeHBwDTm9RoNO32vHq9Hrt378bkyZOhUqna7Xk7GtaBNQBYAwvWgTUAWAOLttZBq9UiNDRU+h5vqQ4RUiyHeDQaTbuHFFdXV2g0GrvfCe29DqwBa2DBOrAGAGtg0V51aO1QDQ6cJSIiIlliSCEiIiJZYkghIiIiWWJIISIiIlliSCEiIiJZYkghIiIiWWJIISIiIlliSCEiIiJZYkghIiIiWWJIISIiIlliSCEiIiJZYkghIiIiWbLrkPLp4Sz8N1OJ1PxyWzeFiIiIrmPXIWVHSj4O5CuRc63a1k0hIiKi69h1SLFcOtoohI1bQkRERNez65CiNGUUMKMQERHJj12HFPakEBERyZddhxT2pBAREcmXnYcU9qQQERHJlV2HFHNGgZEZhYiISHbsOqRYelIEe1KIiIhkx85DiumWPSlERETyY9chhWf3EBERyZddhxT2pBAREclXm0LK6tWroVAo8NRTT91wvS1btqBXr15wdnZG//79sX379ra8bLvhmBQiIiL5anVIOX78ONatW4cBAwbccL3Dhw9j7ty5eOSRR3DixAnMmjULs2bNQkpKSmtfut3Un4Js44YQERFRI60KKRUVFZg3bx7+/e9/w9vb+4brvv3225g6dSqWL1+O3r174+WXX0ZUVBTee++9VjW4PdWfgsyUQkREJDeOrdlo8eLFmDFjBiZOnIh//OMfN1w3ISEBy5Yts1o2ZcoUbN26tdltdDoddDqddF+r1QIA9Ho99Hp9a5rcNHM4qasztO/zdjCW984asAYNb+0V68AaAKyBRVvr0Nb6tTikbN68GT///DOOHz/+q9bPz8+Hv7+/1TJ/f3/k5+c3u82qVauwcuXKRst3794NV1fXljX4BgoLlQCUOHvuHLaXnm235+2oYmNjbd0Em2MNWAML1oE1AFgDi9bWoaqqqk2v26KQkpOTg6VLlyI2NhbOzs5teuEbWbFihVXvi1arRWhoKCZPngyNRtNur7NTm4zkkkL06NkL00eFt9vzdjR6vR6xsbGYNGkSVCqVrZtjE6wBa2DBOrAGAGtg0dY6WI6EtFaLQkpSUhIKCwsRFRUlLTMYDDhw4ADee+896HQ6ODg4WG0TEBCAgoICq2UFBQUICAho9nXUajXUanWj5SqVql13FgelaUiOQqm0653Qor3r2xGxBqyBBevAGgCsgUVr69DW2rVo4OyECRNw+vRpJCcnSz9Dhw7FvHnzkJyc3CigAEBMTAz27t1rtSw2NhYxMTFtanh74CnIRERE8tWinhQPDw/069fPapmbmxs6deokLZ8/fz6Cg4OxatUqAMDSpUsxduxYvPHGG5gxYwY2b96MxMRErF+/vp3eQutZJnNjRCEiIpKfdp9xNjs7G3l5edL9ESNGYNOmTVi/fj0GDhyIb775Blu3bm0UdmxBoeS0+ERERHLVqlOQG4qLi7vhfQCYPXs2Zs+e3daXanfStPhG27aDiIiIGrPza/dwTAoREZFc2XlIMd1yWnwiIiL5seuQolBwTAoREZFc2XVIkc7uYUYhIiKSHTsPKexJISIikiu7Din1h3ts3BAiIiJqxK5DSv3hHqYUIiIiubHzkMKeFCIiIrmy65Bi7kjhmBQiIiIZsu+QIs2TwpBCREQkN3YdUupnnLVxQ4iIiKgRhhSwJ4WIiEiO7DykmG45cJaIiEh+7DqkWOZJYUYhIiKSH7sOKZwnhYiISL7sPKRwTAoREZFc2XVIUXBMChERkWzZdUipPwWZKYWIiEhu7DykmG7Zk0JERCQ/dh1SFByTQkREJFt2HVKU5nfPnhQiIiL5se+QYhmTwpRCREQkOwwpYE8KERGRHNl1SLHgmBQiIiL5seuQUj/jrG3bQURERI3ZeUjh2T1ERERyZechxXTLkEJERCQ/dh1SFBw4S0REJFt2HVI4LT4REZF82XlIMd0yohAREcmPXYcUTotPREQkX3YdUniBQSIiIvlqUUhZu3YtBgwYAI1GA41Gg5iYGOzYsaPZ9Tds2ACFQmH14+zs3OZGtxeOSSEiIpIvx5asHBISgtWrV6N79+4QQuCzzz7DzJkzceLECfTt27fJbTQaDVJTU6X7lkMscsCeFCIiIvlqUUi56667rO7/85//xNq1a3HkyJFmQ4pCoUBAQEDrW3gTcUwKERGRfLUopDRkMBiwZcsWVFZWIiYmptn1Kioq0KVLFxiNRkRFReGVV15pNtBY6HQ66HQ66b5WqwUA6PV66PX61ja5EWE0AAAMBmO7Pm9HY3nvrAFr0PDWXrEOrAHAGli0tQ5trZ9CtHBAxunTpxETE4Oamhq4u7tj06ZNmD59epPrJiQkIC0tDQMGDEBZWRlef/11HDhwAGfOnEFISEizr/Hiiy9i5cqVjZZv2rQJrq6uLWnuDZ0oVmBDmgMiNUY82dfYbs9LREREQFVVFR544AGUlZVBo9G0ePsWh5Ta2lpkZ2ejrKwM33zzDT766CPs378fffr0+cVt9Xo9evfujblz5+Lll19udr2melJCQ0NRXFzcqjfZnB9OXsGfvzmDoWGe+HJRdLs9b0ej1+sRGxuLSZMmQaVS2bo5NsEasAYWrANrALAGFm2tg1arha+vb6tDSosP9zg5OSEyMhIAMGTIEBw/fhxvv/021q1b94vbqlQqDB48GOnp6TdcT61WQ61WN7l9e+4sjo7mt69Q2PVOaNHe9e2IWAPWwIJ1YA0A1sCitXVoa+3aPE+K0Wi06vW4EYPBgNOnTyMwMLCtL9sueHYPERGRfLWoJ2XFihWYNm0awsLCUF5ejk2bNiEuLg67du0CAMyfPx/BwcFYtWoVAOCll17CHXfcgcjISJSWluK1115DVlYWFi5c2P7vpBWUPLuHiIhItloUUgoLCzF//nzk5eXB09MTAwYMwK5duzBp0iQAQHZ2NpTK+s6Za9euYdGiRcjPz4e3tzeGDBmCw4cP/6rxK7eCQupJYUghIiKSmxaFlI8//viGj8fFxVndX7NmDdasWdPiRt0q9TPO2rghRERE1Aiv3QP2pBAREcmRnYcU85gUTpFCREQkO3YdUhS8wCAREZFs2XVIsRzuYUQhIiKSHzsPKTwFmYiISK7sOqQoOJkbERGRbNl1SFFyTAoREZFs2XlIMd2yJ4WIiEh+7DykcEwKERGRXNl1SOGYFCIiIvmy85DCMSlERERyZdchhWNSiIiI5MvOQwrHpBAREcmVXYcUy5gUZhQiIiL5seuQwp4UIiIi+bLzkGK6ZUghIiKSH7sOKfVn99i4IURERNSIXYcUHu4hIiKSLzsPKaZbnoJMREQkP3YeUni4h4iISK7sOqTUn4LMlEJERCQ3dh1SOCaFiIhIvuw8pJhuOSaFiIhIfuw6pCjYk0JERCRbdh1SlJwWn4iISLbsOqSwJ4WIiEi+7DqkcEwKERGRfNl1SLH0pAA8DZmIiEhu7DqkKOszCntTiIiIZMbOQ0p9SuG4FCIiInmx85BS/ztDChERkbzYdUixHpNiw4YQERFRIy0KKWvXrsWAAQOg0Wig0WgQExODHTt23HCbLVu2oFevXnB2dkb//v2xffv2NjW4PbEnhYiISL5aFFJCQkKwevVqJCUlITExEXfeeSdmzpyJM2fONLn+4cOHMXfuXDzyyCM4ceIEZs2ahVmzZiElJaVdGt9W1mNSbNgQIiIiaqRFIeWuu+7C9OnT0b17d/To0QP//Oc/4e7ujiNHjjS5/ttvv42pU6di+fLl6N27N15++WVERUXhvffea5fGt5WCA2eJiIhky7G1GxoMBmzZsgWVlZWIiYlpcp2EhAQsW7bMatmUKVOwdevWGz63TqeDTqeT7mu1WgCAXq+HXq9vbZMbMdTVP1dtrR56h3Z76g7FUtP2rG1HwxqwBhasA2sAsAYWba1DW+vX4pBy+vRpxMTEoKamBu7u7vjuu+/Qp0+fJtfNz8+Hv7+/1TJ/f3/k5+ff8DVWrVqFlStXNlq+e/duuLq6trTJzTId4nE0P3cs3FTt9tQdUmxsrK2bYHOsAWtgwTqwBgBrYNHaOlRVVbXpdVscUnr27Ink5GSUlZXhm2++wYIFC7B///5mg0prrFixwqoHRqvVIjQ0FJMnT4ZGo2m316mtrQWOxAEAJkycCB83p3Z77o5Er9cjNjYWkyZNgkpln0mNNWANLFgH1gBgDSzaWgfLkZDWanFIcXJyQmRkJABgyJAhOH78ON5++22sW7eu0boBAQEoKCiwWlZQUICAgIAbvoZarYZarW60XKVS3bSdxcHR0a53RODm1rejYA1YAwvWgTUAWAOL1tahrbVr8zwpRqPRavxIQzExMdi7d6/VstjY2GbHsNiCAqYBsxw4S0REJC8t6klZsWIFpk2bhrCwMJSXl2PTpk2Ii4vDrl27AADz589HcHAwVq1aBQBYunQpxo4dizfeeAMzZszA5s2bkZiYiPXr17f/O2klBQABTuZGREQkNy0KKYWFhZg/fz7y8vLg6emJAQMGYNeuXZg0aRIAIDs7G0plfefMiBEjsGnTJjz33HP461//iu7du2Pr1q3o169f+76LNlCYUwp7UoiIiOSlRSHl448/vuHjcXFxjZbNnj0bs2fPblGjbiXLTCmczI2IiEhe7PraPYC5JwWAkSmFiIhIVhhSzLc82kNERCQvDCmWnhSmFCIiIlmx+5BiKQBDChERkbzYfUjhwFkiIiJ5YkgxpxTBnhQiIiJZYUgx37InhYiISF4YUjhwloiISJYYUsy3DClERETywpAijUmxbTuIiIjIGkOK+ZYhhYiISF4YUsy3PNxDREQkLwwpHDhLREQkSwwp5luegkxERCQvDCmczI2IiEiW7D6k1F+7x6bNICIiouvYfUjhmBQiIiJ5Ykgx3zKkEBERyQtDivmWGYWIiEheGFJ4uIeIiEiWGFLMtxw4S0REJC8MKexJISIikiWGFPMt50khIiKSF7sPKUpLT4rRtu0gIiIia3YfUngKMhERkTwxpFimxbdtM4iIiOg6dh9SLDgmhYiISF7sPqTw2j1ERETyZPchhacgExERyRNDink0CntSiIiI5IUhxTJwlj0pREREssKQYr7l4R4iIiJ5aVFIWbVqFYYNGwYPDw/4+flh1qxZSE1NveE2GzZsgEKhsPpxdnZuU6PbkxRSOJkbERGRrLQopOzfvx+LFy/GkSNHEBsbC71ej8mTJ6OysvKG22k0GuTl5Uk/WVlZbWp0e+LAWSIiInlybMnKO3futLq/YcMG+Pn5ISkpCWPGjGl2O4VCgYCAgNa18Carv3aPTZtBRERE12lRSLleWVkZAMDHx+eG61VUVKBLly4wGo2IiorCK6+8gr59+za7vk6ng06nk+5rtVoAgF6vh16vb0uTrej1eunaPfq6unZ97o7E8r7t9f0DrAHAGliwDqwBwBpYtLUOba2fQrTytBaj0Yi7774bpaWliI+Pb3a9hIQEpKWlYcCAASgrK8Prr7+OAwcO4MyZMwgJCWlymxdffBErV65stHzTpk1wdXVtTXOb9dF5JU5fU2JOhAEj/NmdQkRE1F6qqqrwwAMPoKysDBqNpsXbtzqk/PGPf8SOHTsQHx/fbNhoil6vR+/evTF37ly8/PLLTa7TVE9KaGgoiouLW/Umb9SW3723F6euKrHyrt54YHhouz13R6LX6xEbG4tJkyZBpVLZujk2wRqwBhasA2sAsAYWba2DVquFr69vq0NKqw73LFmyBD/88AMOHDjQooACACqVCoMHD0Z6enqz66jVaqjV6ia3be+dxTImRalU2vWOCNyc+nY0rAFrYME6sAYAa2DR2jq0tXYtOrtHCIElS5bgu+++w759+xAeHt7iFzQYDDh9+jQCAwNbvO3NUH92j23bQURERNZa1JOyePFibNq0Cdu2bYOHhwfy8/MBAJ6ennBxcQEAzJ8/H8HBwVi1ahUA4KWXXsIdd9yByMhIlJaW4rXXXkNWVhYWLlzYzm+ldTiZGxERkTy1KKSsXbsWADBu3Dir5Z9++ikefPBBAEB2djaUyvoOmmvXrmHRokXIz8+Ht7c3hgwZgsOHD6NPnz5ta3k7qQ8pNm0GERERXadFIeXXjLGNi4uzur9mzRqsWbOmRY26lXjtHiIiInnitXvMt8woRERE8sKQwmnxiYiIZIkhxXzLMSlERETywpBivmVPChERkbzYfUhRcuAsERGRLNl9SOHhHiIiInliSOHAWSIiIlliSDHfsieFiIhIXhhSOCaFiIhIlhhSzLc83ENERCQvDCm8CjIREZEsMaSYb9mTQkREJC92H1IsBWBGISIikhe7DymWrhQjj/cQERHJit2HFJ6CTEREJE8MKZZTkMGUQkREJCd2H1I4JoWIiEie7D6k8OweIiIieWJI4bV7iIiIZIkhxTwWhQNniYiI5IUhhdfuISIikiWGFPOt0WjTZhAREdF17D6kOJhTit7AlEJERCQndh9S1A6m2wpdnW0bQkRERFbsPqQ4m0NKZS1DChERkZzYfUip70kx2LYhREREZMXuQ4qlJ6WiRm/bhhAREZEVhhQH06nHlexJISIikhW7DykcOEtERCRPdh9SGg6c5YRuRERE8mH3IcXSkyIEUFXLQz5ERERyYfchxUkJKM0TuvGQDxERkXy0KKSsWrUKw4YNg4eHB/z8/DBr1iykpqb+4nZbtmxBr1694OzsjP79+2P79u2tbnB7UygAN7UjAIYUIiIiOWlRSNm/fz8WL16MI0eOIDY2Fnq9HpMnT0ZlZWWz2xw+fBhz587FI488ghMnTmDWrFmYNWsWUlJS2tz49uLmZDrmU8mQQkREJBuOLVl5586dVvc3bNgAPz8/JCUlYcyYMU1u8/bbb2Pq1KlYvnw5AODll19GbGws3nvvPXz44YetbHb7clc7AtChooYhhYiISC5aFFKuV1ZWBgDw8fFpdp2EhAQsW7bMatmUKVOwdevWZrfR6XTQ6XTSfa1WCwDQ6/XQ69tv0jXLc7mae1LKqnTt+vwdheU92+N7t2ANWAML1oE1AFgDi7bWoa31a3VIMRqNeOqppzBy5Ej069ev2fXy8/Ph7+9vtczf3x/5+fnNbrNq1SqsXLmy0fLdu3fD1dW1tU1uVk1FKQAlDh1LQm2m/Z6GHBsba+sm2BxrwBpYsA6sAcAaWLS2DlVVVW163VaHlMWLFyMlJQXx8fFtakBTVqxYYdX7otVqERoaismTJ0Oj0bTb6+j1esTGxqJLoD8ulBUhsldfTI8Oa7fn7ygsdZg0aRJUKpWtm2MTrAFrYME6sAYAa2DR1jpYjoS0VqtCypIlS/DDDz/gwIEDCAkJueG6AQEBKCgosFpWUFCAgICAZrdRq9VQq9WNlqtUqpuys7i7mJ6zug52vTPerPp2JKwBa2DBOrAGAGtg0do6tLV2LTq7RwiBJUuW4LvvvsO+ffsQHh7+i9vExMRg7969VstiY2MRExPTspbeRO7SKcj2feyRiIhITlrUk7J48WJs2rQJ27Ztg4eHhzSuxNPTEy4uLgCA+fPnIzg4GKtWrQIALF26FGPHjsUbb7yBGTNmYPPmzUhMTMT69evb+a20nrt0CjJnnCUiIpKLFvWkrF27FmVlZRg3bhwCAwOln6+++kpaJzs7G3l5edL9ESNGYNOmTVi/fj0GDhyIb775Blu3br3hYNtbjZO5ERERyU+LelJ+zQX44uLiGi2bPXs2Zs+e3ZKXuqXczRfw4TwpRERE8mH31+4B6ntSKmsZUoiIiOSCIQUNB84ypBAREckFQwoANx7uISIikh2GFABuTubDPexJISIikg2GFAAaF1NIKa3mPClERERywZACwM/DNLttVa0B5TUMKkRERHLAkALA1ckRHs6m3pQCbY2NW0NEREQAQ4okQOMMAMgv09m4JURERAQwpEgCPM0hhT0pREREssCQYuZv7knh4R4iIiJ5YEgxqz/cw5BCREQkBwwpZv483ENERCQrDClmATzcQ0REJCsMKWY83ENERCQvDClm/p6mCd2KK3SoMxht3BoiIiJiSDHzdVPDUamAUQBFFZwrhYiIyNYYUsyUSoU0PT4P+RAREdkeQ0oDgV4uAIA8hhQiIiKbY0hpINB8GnJuabWNW0JEREQMKQ0Em3tSckvZk0JERGRrDCkNBEkhhT0pREREtsaQ0oB0uKeMIYWIiMjWGFIaCOLhHiIiItlgSGnAElKKK3So0Rts3BoiIiL7xpDSgLerCs4qU0k4VwoREZFtMaQ0oFAo6g/5cFwKERGRTTGkXCfIk+NSiIiI5IAh5TpBXqYzfK5cY08KERGRLTGkXKe7nwcA4ETONRu3hIiIyL4xpFxnZKQvAOBoxlXo6niGDxERka0wpFynV4AHfN2dUK034ER2qa2bQ0REZLcYUq6jVCqk3pT4tGIbt4aIiMh+tTikHDhwAHfddReCgoKgUCiwdevWG64fFxcHhULR6Cc/P7+1bb7pRplDyvenclFeo7dxa4iIiOxTi0NKZWUlBg4ciPfff79F26WmpiIvL0/68fPza+lL3zKT+wSgs4caWSVVePLLExBC2LpJREREdsexpRtMmzYN06ZNa/EL+fn5wcvLq8Xb2YKnqwqfLBiG+z48jLjUIqQVVqCHv4etm0VERGRXWhxSWmvQoEHQ6XTo168fXnzxRYwcObLZdXU6HXQ6nXRfq9UCAPR6PfT69jv8Ynmupp6zl78revi5IyVXi4wCLcJ9nNvtdeXmRnWwF6wBa2DBOrAGAGtg0dY6tLV+CtGGYxkKhQLfffcdZs2a1ew6qampiIuLw9ChQ6HT6fDRRx9h48aNOHr0KKKioprc5sUXX8TKlSsbLd+0aRNcXV1b29wW+zRVieSrStzT1YBxgTzkQ0RE1BJVVVV44IEHUFZWBo1G0+Ltb3pIacrYsWMRFhaGjRs3Nvl4Uz0poaGhKC4ubtWbbI5er0dsbCwmTZoElUrV6PF/7bqAj+IvYf4dYfj7jF7t9rpy80t1sAesAWtgwTqwBgBrYNHWOmi1Wvj6+rY6pNyywz0NDR8+HPHx8c0+rlaroVarGy1XqVQ3ZWdp7nm7+roDAHLLauxiJ71Z9e1IWAPWwIJ1YA0A1sCitXVoa+1sMk9KcnIyAgMDbfHSLRLqYzq0lHOV1/EhIiK61Vrck1JRUYH09HTpfmZmJpKTk+Hj44OwsDCsWLECV65cweeffw4AeOuttxAeHo6+ffuipqYGH330Efbt24fdu3e337u4SUK9TVdEzr5aBSEEFAqFjVtERERkP1ocUhITEzF+/Hjp/rJlywAACxYswIYNG5CXl4fs7Gzp8draWjz99NO4cuUKXF1dMWDAAOzZs8fqOeQq2NsFCgVQrTegpLIWvu6ND0ERERHRzdHikDJu3LgbTm62YcMGq/vPPPMMnnnmmRY3TA7Ujg7w93BGvrYGOVerGFKIiIhuIZsMnO1Iwnxcka+twcmcUry+OxVF5Tr8dkgIFo2O4OEfIiKim4gh5Rf0CHDHsUtX8eL3Z6Vlr2w/Dy8XJ9w/LNSGLSMiIrq98SrIv+BPE7ojKswLAODj5oS7BwYBAHaeMV0g8VJxJbYlX+H1fYiIiNoZe1J+gZ+HM756LAY7U/IxOMwLFbo6/O9kLg6lF6O61oDFm37GmVwt3NWOmNDb39bNJSIium2wJ+VXUDkocdfAIIR4u6KnvweCvVygqzNi8/FsnMk1XVfo8MUSG7eSiIjo9sKQ0kIKhQITevsBAFY2GKdy/NJVWzWJiIjotsSQ0gr3RoXg+hN7zuRqUamrs02DiIiIbkMMKa0wKNQLj46OkO57ODvCYBQ4kV1qu0YRERHdZjhwtpWWTe6BonIdfD3UKNTWYGtyLvacK8Co7r62bhoREdFtgSGlldSODnhzziAAwM6UPGxNzsWGw5eQUVwJjbMjXvvtQLg4Odi2kURERB0YD/e0g6n9AvHIqHAAwIELRfjhVB5+OJVr41YRERF1bAwp7eRv03vj77/pAzdz78n/TuZCCIELBeW4VFxp49YRERF1PDzc006USgUeGRWOCb38MO71OBxMK8bAlbuhramDi8oB+5ePg5/G2dbNJCIi6jDYk9LOuvq6oX+wJwBAW2M6Jblab8CRTM6jQkRE1BIMKTfBkjsj4eWqwoKYLpg9JAQAkHTdZG85V6uwZNPPOHyx2BZNJCIikj0e7rkJpvQNwJS+AQCA70/mYkvSZSRlXwMA6OoMuHKtGvd8cBhl1Xqcy9Ni79PjbNhaIiIieWJIucmGdvUGAJzLK0dS1jU8tjEJxRU66fGLRZUoq9LD01VlqyYSERHJEg/33GSBni4I8nSGwShw39rDKK7QQakAnFX1pY9P5yEfIiKi6zGk3AJ3dOsk/T4w1Asnnp+MlBenWM2tsjMlD9Gv7MGPp/Js1UwiIiJZ4eGeW+Av03phUKgXunRyw4hunaByMGXD0d198XF8Jnak5OGrxBwAwPqDGZgxINCWzSUiIpIF9qTcAn4ezpgf0xVje3SWAgoA3BHRCT383aVTlQGgxDxeZfeZfIx59SfsOVtwy9tLREQkBwwpNuSscsDWxSOxfEpP3NnLDwBwpbQaJRU6PLc1BdlXq/DUV8mcsZaIiOwSQ4qNuTo5YvH4SHzy4DB0cnOCEMCz/z2FwnJTj0qFrg5//e60jVtJRER06zGkyEgPfw8AwJ5zhQCAZ6b2BAAkZJSgpEKHqto6GI3CZu0jIiK6lRhSZKRngIf0e5iPKx4f0w19AjUQArjng8Po8/wu9Pz7Dqw/cNGGrSQiIro1GFJkpLu/u/T73OFhUCoVmNjbNFYl+2oVAEBvEPgm6bJN2kdERHQrMaTISGTn+pAye6jpmj8TevtLyyI6uwEAMosrUWcw3trGERER3WIMKTIyrKsPHhsTgdd+OwC+7moAQP9gT0T6ucNd7YiP5g+Fs0oJvUEg51q1tF1+WQ3KqvS2ajYREdFNwcncZESpVGDF9N6Nln37xAjo9EZ09lAjwtcdZ/O0WLX9HC4WVUDloMT5/HKE+7oh9s9j4OjA3ElERLcHhpQOQOOsApxNv0f6mULK7usmecssrsShiyUwCoGYiE5wVjnYoKVERETth392dzCRfu5W99/+3SBpIrgnvkjCQ58ex5NfnoAQPFWZiIg6NoaUDqZhSJnePwAzBwVjoflChZW1BgBA7NkCbOEZQERE1MG1OKQcOHAAd911F4KCgqBQKLB169Zf3CYuLg5RUVFQq9WIjIzEhg0bWtFUAoBuDc4AmtI3AAAQHdEJvu5OAEzzqwDA67tS2ZtCREQdWotDSmVlJQYOHIj333//V62fmZmJGTNmYPz48UhOTsZTTz2FhQsXYteuXS1uLAFdfV3h42YKJON6mg7zOCgVWH3vACyI6YKti0fCWaVEYbkOFwoqbNlUIiKiNmnxwNlp06Zh2rRpv3r9Dz/8EOHh4XjjjTcAAL1790Z8fDzWrFmDKVOmtPTl7Z7a0QHfPzkKRqOAp4tKWj6xjz8m9jHNqTKsqw8OphXjUHqx1Sy2REREHclNP7snISEBEydOtFo2ZcoUPPXUU81uo9PpoNPppPtarRYAoNfrode333wgludqz+e8FfzcTP9szbU7uqs3DqYVY++5AvQLckd4Jzd4uaqaXLfh83S0OrQn1oA1sGAdWAOANbBoax3aWr+bHlLy8/Ph7+9vtczf3x9arRbV1dVwcXFptM2qVauwcuXKRst3794NV1fXdm9jbGxsuz+nLYkKAHDEoYslOHSxBAoITA4RmB5641lqb7c6tAZrwBpYsA6sAcAaWLS2DlVVVW16XVnOk7JixQosW7ZMuq/VahEaGorJkydDo9G02+vo9XrExsZi0qRJUKma72noaAxGgY/T41BabUqwAgoklarx3mPjYDQK7DhTgGAvZwwM8YRCobht69ASrAFrYME6sAYAa2DR1jpYjoS01k0PKQEBASgosJ54rKCgABqNpsleFABQq9VQq9WNlqtUqpuys9ys57UVFYA35wzEqctleGB4GO5YtRfFFbW4VmPA18dz8PruCwCAmIhO+M/C6PrtbrM6tAZrwBpYsA6sAcAaWLS2Dm2t3U2fJyUmJgZ79+61WhYbG4uYmJib/dJ27c5e/nhqYg/4aZwRYT5t+YeTeXhnX7q0TkJGCa6UVjf3FERERDbV4pBSUVGB5ORkJCcnAzCdYpycnIzs7GwApkM18+fPl9Z//PHHkZGRgWeeeQbnz5/HBx98gK+//hp//vOf2+cd0C/qG2Q6RPbSD2dRW2fEmB6d0cPfFFzSi3iaMhERyVOLQ0piYiIGDx6MwYMHAwCWLVuGwYMH4/nnnwcA5OXlSYEFAMLDw/Hjjz8iNjYWAwcOxBtvvIGPPvqIpx/fQn0Crcfx/GVqL3T3M52anM65VIiISKZaPCZl3LhxN5zJtKnZZMeNG4cTJ0609KWonfQJqg8pA0I80SdIg27m6fXTCytQoK3BmtMO+KnqNN6aG2WrZhIREVnhtXvsQMOelFmDggEA3c0h5XjWVfzhk0RcqlBg68k85JVxjAoREckDQ4od6OSuxrienRHR2Q2zh4YAqL9QYUZRJTJL6s9jT7hYAgAQQuCLI1n4xw9nYTTyGkBERHTryXKeFGp/Gx4aDiEEFAoFACDc183q8RA3gcuVChy+WIJ7Bgdj9c7zWLc/AwAwrX8AhnTxueVtJiIi+8aQYkcsAQUAnFUOVo/dFWbE2nMO+PFUHmrrjPjfyVzpsfTCCoYUIiK65Xi4x47Nj+kCR6UCGx4cgggPAUelAtV6A/53MheOSgXcnExBJr2wAkIILPwsEfd+cAi1dTeeXp+IiKg9MKTYsed/0wdH/zoBI7t1gpMDMLmPHwCgf7AnvlgYjRXTewMALhZV4kBaMfacK8DP2aXIKOZpy0REdPPxcI8dc3RQopO7WrpK5eu/7Y+XZ/VHJ3fTJQksZ5pfLKrAx/GZ0nZ5ZTXoFdB+11AiIiJqCkMKSVQOSrg6119noZufaXBtVkkVshqcAZRXWnPL20ZERPaHh3uoWZ3d1dA4N86xnEuFiIhuBYYUalbDs4EAYMaAQABArrknxWAUyC+rQUmF7pa3jYiIbn8MKXRDQ7p4AwACPZ0xoZdpYG2+thqVujpMeesA7li1F0P/uQcpV8ps2UwiIroNcUwK3dDzd/VFmE8mHh/XDZeKTeNS8kpr8N5P6UgvNJ3lIwRwJKME/YI9bdlUIiK6zTCk0A2F+7ph5cx+AACd3jQ/SkZxJT46aJqNtqe/B1ILynGppNJmbSQiotsTD/fQrxbg6Sz9rjcIjO7ui0dGhwOA1MtCRETUXhhS6FdzVjnA06X+FOX5MV2lawBlFleiRm/Ayz+cxdz1R1Beo7dVM4mI6DbBkEItUlZdHz7G9eyMrp1MIeVKaTVmvX8IH8dnIiGjBAfTiqX1UvPLsXTzCeSW8tRlIiL69TgmhVqkaydXXCqpwh0RPlA5KOHr7gR3tSMqdHU4n18urXexsH7q/ClvHQAA1BkE3p8XdcvbTEREHRN7UqhF3p8XhQdHdMWHvx8CwDSXSldf10brpReZQkrDid9yrnHcChER/XoMKdQifYM88eLdfeHl6iQt8/eoH1C7eHw3AJBOT956Ild6TG8Qt6iVRER0O2BIoTZrODHtPYNDAAAZRZUoqdDhiyNZ0mNZJZUQgkGFiIh+HYYUarOnJvaAr7sT/nVff3Tp5AqVgwLVegOmv3MQV0qr4edhuqpyVa0BReU6GIwC6YUVDCxERHRDDCnUZv2CPZH43CTMGRYGlYMSXcxn/BRodfB0UWHTomiE+rgAMJ2q/MR/kjDxzf2ISy2yZbOJiEjmGFKo3bk5OUi///Oefoj085BOVX53Xzp2nSkAAOw7XyitdzZXi4WfJeJoRsmtbSwREckWT0Gmdje5bwBOXi5DrwAPzOhvunJy105uOJhWjPj0+vlTCrSmqylnFFVg+jsHAQC6OgOiIzoBAIQQja7ETERE9oMhhdrdgyO6ItDTGZP7Bkgho0unxqcpW2apfXRjkrTsbK4WABB7tgCLN/2Mt+YMwnRz0CEiIvvCwz3U7tzUjrg3KgTu6voM3DPAQ/p908JoAEBWSRVe3Zkqna4MANeqalGjN2DR54morTNi2dfJt6zdREQkLwwpdEuM7OaLlXf3xQ9PjkJ0RCc4OSpRazDik0OZAICPFwyFp4sKRgGcMfemAIBKWb+L5pfV4P4PE/B5wqVb3XwiIrIBhhS6JZRKBRaM6Ip+wZ5wUCrQtcHhn6gwL0zo7Y/ufu4AgM8OX5Iec3CoH5OyeNPPOHbpKp7fduaWtZuIiGyHIYVsIsLXXfp9xoAgAEB3f9Oy/52sn6W2tEqPsio90gsrkJR1TVpeXWu4RS0lIiJbYUghm3BV15+mPL1/AACgW2f3JtfNulqJdfsvWi27aL42UFG5DnUG401qJRER2RJDCtlEdLgPAMBF5YBAT9NEb9396wfX3hHhg6gwLwDA+fxybD+dBwBQmo/+XCgoR1xqIaJf2YNVO87fuoYTEdEtw5BCNvHbIaFYfW9/xC4bIy3rF6SBs0oJD2dHvDVnMMLNh4Q+PpiJyloDQrxdMHd4GADgQkEFHvz0OIwC+Dg+0ybvgYiIbq5WhZT3338fXbt2hbOzM6Kjo3Hs2LFm192wYQMUCoXVj7Ozc7Prk31wUCrwu+FhCPGuH0DbyV2NHUvHYO/TYxHg6SzNrZJaUA4AuGdwsHQq8+bj2VbPx0M+RES3nxaHlK+++grLli3DCy+8gJ9//hkDBw7ElClTUFhY2Ow2Go0GeXl50k9WVlaz65J9C/d1g5+HKcQ2nADOUanAvVEh6O5nCimlVXqr7XKuVUMIgaSsqxxUS0R0m2hxSHnzzTexaNEiPPTQQ+jTpw8+/PBDuLq64pNPPml2G4VCgYCAAOnH39+/TY0m+9AnUCP9/sG8KIT7uqGHf/3gWh83J2icTRPGXSyswLv70nHf2gS8sTvV6nkKy2tQo2dwISLqaFo0LX5tbS2SkpKwYsUKaZlSqcTEiRORkJDQ7HYVFRXo0qULjEYjoqKi8Morr6Bv377Nrq/T6aDT6aT7Wq1pci+9Xg+9Xt/cZi1mea72fM6OSK516OrjjI0PDUWglzO6+LhCr9dDo1ZifE9f5JbW4P0HBuGtPen44XQ+9pzLx7cnTKcuH7hQJL2XEzml+MMniRjW1RufLhjS7GvJtQa3EmtgwjqwBgBrYNHWOrS1fgohhPi1K+fm5iI4OBiHDx9GTEyMtPyZZ57B/v37cfTo0UbbJCQkIC0tDQMGDEBZWRlef/11HDhwAGfOnEFISEiTr/Piiy9i5cqVjZZv2rQJrq6NrwFD9kcIQKEAduYosOOyg9VjCgisHmaAoxJ4LtEB1QbTKUFvRNfBkUPFiYhumaqqKjzwwAMoKyuDRqP55Q2uc9MvMBgTE2MVaEaMGIHevXtj3bp1ePnll5vcZsWKFVi2bJl0X6vVIjQ0FJMnT27Vm2yOXq9HbGwsJk2aBJVK1W7P29F05DooUvKx46tTAEzjVtzVjiit1iOgbzR+Si1CtaF+gG3E4FHQuDhi8aZk/GZAIB4bEy49diC1AKeTk/DovR2vBu2lI+8H7Yl1YA0A1sCirXWwHAlprRaFFF9fXzg4OKCgoMBqeUFBAQICAn7Vc6hUKgwePBjp6enNrqNWq6FWq5vc9mbsLDfreTuajliHHoGe0u+PjApHblkNvj+Zi39sT0VagwsXAsD5wkocTCvC+YIKXD6QiYVjusFZ5YBtyVewdPNJOCkd8PA9Srh2sBq0t464H9wMrANrALAGFq2tQ1tr16LObycnJwwZMgR79+6VlhmNRuzdu9eqt+RGDAYDTp8+jcDAwJa1lKgJEb7uCPd1Q6SfO56c0B2DQr0AQAooz83ojcfGRAAANh3NxvbT+QCACl0d9l8oQlpBOZ75xtQTU2tUIMV8ccPc0mqUVtVavVZJhQ45V6tuxdsiIiK04nDPsmXLsGDBAgwdOhTDhw/HW2+9hcrKSjz00EMAgPnz5yM4OBirVq0CALz00ku44447EBkZidLSUrz22mvIysrCwoUL2/edkF1yclRiz7KxMBgFnByVGGyepRYAloyPxMLREdiWfAUAcPpKGQDAWaVEjd6IH0/lwc9DDV1d/RwrP2eXorPGBXe9F48uPm7Y+dRoKBQK6OoMmPn+IZRU1GL/8nHw0zgj5UoZag1GRIV539L3TERkL1ocUubMmYOioiI8//zzyM/Px6BBg7Bz507ptOLs7GwolfUdNNeuXcOiRYuQn58Pb29vDBkyBIcPH0afPn3a712QXXNQKuBgni9/UIgXHhzRFX4aNf44thsAoG9Q/SEhVycHrJkzCI9tTMKecwXwdnUCAAzt4oXErFKcyClFYnYpavRGpBaUI6ukCl193fDtz1dw+Vo1AOD4pWsY08MXc9YloM4okLBiAnzcnG7xuyYiuv21auDskiVLsGTJkiYfi4uLs7q/Zs0arFmzpjUvQ9RiSqUCL95tfXp7uK+b9PuTd3bH5D7+6O7njrTCClTVVkPtqMST47thwYYk7D1fZLXtkYwShPq4Wl3gMCXX1CNTaZ40LjnnGu7sZQrpqfnlKK2qRXREp5vy/ojIfuWX1cDRQQFf98ZjNlviZE4pjmaW4JFREdIfeHJ108/uIbI1B6UCH/4+CheLKvHomAgoFAqsmN4LD29IBACM7u6LIV2aPmSTkFGCtMIKXCqpH4ty+nIZCspqpPvJOWUY3b0zFn6WiP0XTCHnPwujMTLS9ya+KyLqqOoMRry2KxVdfd0wd3gYDEaBTUez0CfI0+qz6MCFIhiEwPiefiip0GHKWwfgrnbE/uXj4OhgOmJRozdAW62Hn8Y0U3fO1SqUVevRL9jUg1xaVYtNx7KRUVSJ52b0hoNSgYc3HEdJZS06e6hxz+AQFFfocKGgHCO6ye8ziyGF7MLUftYDtcf39MOYHp1x4EIR7h4UDLWjEj08jbhQpsT4np3xh5gueHhDIrYl50rbPDY2Auv2Z+BkTimUDf76OJlTit1nCqSAAgA7U/IxMtIXpy6X4kJBBe6LCoZCYdomPq0Ym49n46/TeyPIy+Umv3MikpudZ/Kx7kAGHJUK/GZAIL5JuoyV35+Fv0aNhL9MgEIB/GtnKj7cfxFKBXDw2Tux43Qeyqr1KKvWI72oAr0CNMgrq8a8fx/F5WvV+PFPo+DpqsJv3o1Hpa4O+54eB09XFWa8E48rpaZD1X4eaigUQEml6aSAHafzMaZ7Z8z791FkFlfi3wuGYmyPzrYsTSMMKWSXFApT78rpy2UYHu6Duro6zOtmhF+vKEzuFwT9dRcsXD6lJxaNjsCn8ZdQrqsDACgVgFEAJy+Xoto87X7fIA3O5GoRd6EQ+WU1uPu9QwCAUG8XREd0wk/nC/HQhuMAgDAfVzwztVezbRRCSMGG6FYxGH/1/J7t7p29aSgq12Hl3X2t/hBo6GBaEY5lXsXSCd3h6KBEpa4ONXoDOrXwEIgQAmfztOjp7yH1SjSlutYAR0fHX/1/sUJXh9d3pWJ8L79mv/A/PXQJAFBnFPhv0mW8sfsCAKBAq8OpK2U4n6fFh+ZDzEYB7D6Tj29/viJtn5xdikCNC+asO4Js8xmHnyVcQklFLcqqTTO87j6bj8vXqqWAAgBfHc9BhfnzCwD2XyjCvI+OIrWgHH4eaoR6y++PJs6/SXbL1ckR0RGdpA8fLzUwsbcfHJQKOKscMHd4GDp7qPHBvCgsHh8JJ0clAjzrr+D9yKhwODkoUVqlx7HMq3BQKvDu3MFwclAi52o1fv9x/QzMB9OKkVVSicWbfpaWHb90FQCw4VAm7nw9DukN5nX538lc9Hl+FzYfs77ac3uo1NXxqtEdSHphOb4+noMWTA7eascyr2LQS7vxZmxas+vo6gx49ptT2JhwqV1fO7e0Gm/GXsDGI1nSuK/r1RmM+PNXyXh3Xzp2ny2AEALzPjqKsa/FIbfBl3FGUQU+OpjR7DW7hBD4vy2nMOOdeKw7kAEAuFJajbf2XJC+5AHgcIECQ17Zh1e2nwMA5JVV4+9bU3Aur/kJyt7YnYoNhy/hb9+dRmF5Dd7Zm4bHNyYhPq0YAJCcU4qkrGvS+i//eM4qOGw9cQWvm68/Zhl7snrHeZxt8JonL5file3npIACAF8cycaOlHzp/tt70vC5+d/okweHwstVhZLKWujqjBgZ2QlhPq7Q1RlxPt8UUL589A5EdK6/NppcMKQQNWPVvf1x7K8TML1//aGiGQNMv98bFYwV03qjT1D9DMh3DwxCRGd3DA/3AQCr0BGfXow/f5WMqlqDdCZQyhUtiit0eHVXKjKKK6XBuZeKK/GnL0+gWm/Ai9+fgRACRzNK8Mr2cyiuqL+mVWu+tC4WVWD4P/dg2dcnW7zt9W7FlyYBSzadwDP/PYW955q/0nx7qNDV4f51CSivqcPaA5nS8hq9werfemdKPr5KzME/fjzXqMfxRs7na5HdYGxXan45Csvrx3btPlP/BZt46RqKK3T43foEPLzhOIzm3p2jmVdRXGE6VJFwsQSnLpchOacUFbo67D1XYN72Ku58Yz/+8eM5bEzIarItH8RdxH9/vgwA2JKYg9o6I+5+Nx5v7UnD+z+ZJhr95ucr+CrDAXqDwL8PZqK0qhb3fnAYG49k4YVtZwAA2ho9dp/JR615GoMLBeX43Pyal6+ZDsW8GXsBO8/k4+/bUlCpq8PyLab/exHmAf0Go4BSASwabZoBe8PhSyiuqEWErxv++0fT/GOWaRLU5ut6fHksB18l5pja/3gMQhr0gCyI6QIAKNfVwSiA3wwIxJ29/DFzYBAAwM3JAf+6bwCm9TNNwNrZHFC6yTCgAAwpRDd0fRfvUxO744cnR+GN2QOhVCrw50k9MKGXH168qw9W3dsfAHDXQFOQ8XV3wvIpPQGY/nr6ObsU7mpHbFs8Eh7OjqjWG/DXb0+jynyW0A+n8nAmtwwPmw8HAUCN3og5649gzvojWH8gQ7rC8/oDF9HjuR04lF4srZt46SqiX9mDL440/cEMAF8n5qCy1oCdDT5Ym1JVW3fDLyCDUeCBfx/F5DX7UV3b/BWmNxzKxMMbjlv9dfpL/pt0Gb//6ChKGgSy1jAaBfZfKEJ5TfOvbTAKfHU8G2kF5W16rZslo6gC5/NNbUs0//W9asc53PPBIauabj6WjQEv7kJS1tUbPl9eWbX0hZ9wsQR3vxePE9mm5313r3XvSU0dcPhiCfo8vxNvN3jMcthBV2dEan459AYjPjqYYdU7cL30wgrc/e4h/PbDw6gzGJGcU4rp7xzE79YdkQ4v7TpTP5P5nnMFuH9dAo5kXMW+84U4klECAPjhVP0YsSMZJdiaXH8IZP+FYlTq6rDw80Rp2e6zpuAjhEBVram3Ir+sxur9ZF+twr92npfGaWw+lo2q2jq8uuuC1XsY9s89yDMPmD926Sp+Ol+I37wTj0c3JmHVjnMQQuCl789aHS5rOOt1ZnElHvjoKNIKK+DnocZ/FkXDXW0acTE/pisWj4+UzrRRKoCXZ/VDl05u6G8eANvZQ41tS0Zaten3d4RhWFcf/GVaL3ioHfHcjN5YObMfnBocvlppPtvx8XHdML5nZ7z9u8EI8XbFE+Mi8X+Te+C/j4+QbUABGFKIWkTt6IB+wZ5SeBnbozM+fnAYHhwZDmeV6UKH9w8NxQ9PjsL+5eNNh4kafGCsmN4LoT6u0sy4u8+aPpidVUpU6w2Y8U48MoorEeTpLK1zLLP+i2fvuUIUaGvwZuwF6A1COm5dZzDitx8moECrw3NbU5psuxACP5zMAwDU1hlxJrcMGUUVmLv+CL46Xn9Y6UyuFiNX78Pd7x2CEAIGo8Bbey5gx+k8aZ0fT+chIaMEFwoqcDSzpMnXy7lahX/8eA77zhdi6wnTl0mlrg5F5c2Hj6JyHZ7echLx6cXYdPTXH+oq1NZg87Fsq8NYb+25gAWfHMPqHeeb3e7dfWl49r+n8afNydKyrJIq6H+hgyDlSplVLwBgCqIN/61+jY8OZuDvW1OaDYSW/QMATl0uRXZJFdYfyMCJ7FJsN/97CCGw7kAGtDV12HQ0R1r/+rElnydcQsyqfXjvp3QIIbDy+zM4dbkMb+1Jg9Eo8N2JK1br51YBGxKyYBTAx/GZqNEbUFheg4Np9QPET+SU4p8/nsM/fjyHRz47jtKqWugNRmxLvmI1FuLD/RdRazCisNw05mJN7AUYjAIZxZXYd74QReU6HLtUX7vDF0uQUVQp3f/m58soLK+xOpyRVlghje0AgISLxdiSmIPSKj1UDqb/n0lZ10z7+L+PYNBLsdiWfAUfxKWjts6I4V19EOHrJr0/C21NHV783xlcq9LDVy3wxFjTjNV6g6nHw+KhDcelwy1bEi/jfydzEZ9eDCdHJeabezMAYFSkL347xHQx3ZM5pXBQKvDeA1EI9HTB33/TG/cODsbTk3vAy9UJf5/RG78dEoI9y8ZKZweunNkXC2K64McnR6FXQH3PrcpBgb9O7w0A+M2AIJx6cTIWjo6QttE4O+LjBUOlsTqBni749KHhmNjHNF2Cp6sKS+7sjrBO8r5oL0MKUTtTKBToF+wJN/NfSVPN3arerirMHRYGABjcYJbaqDAvLJ9SP4B2UKgXti4eicfNH44A8NfpveDq5IDCch0e25iEGvO36MG0YuRcrcKX141dOZNbhue2nsbI1fvw46k8GIwC20/nW31x7DlXgIc2HEdCRgme33YGuaXVKKkBHvosCdeq9DiXp8X5/HLsO1+It/akYdnXJ1Fda4DBKPBOg79EEy6WmL4o91/EJw0+7N/em4Y68xfl9tN5qK414J4PDmHMqz9ZjR9oeCjB0tUOQDoGL4RoNLbg+5O5mP3hYcSav8SXbk7GX749jf+Yg422Ro939pme6z9Hs5s8NFWhq8Nbe0zv41yeFjV6A7Yk5mDiW/H4JrP+o7Gqtg7v/5SOnSmmUHAypxR3vxePhz6t7/FKzS/H7A8PY+6/j+BSsenLNa2gHFklpt9LKnSY/vZBPLX5hLRNWkG56ZDEkSzsMh/qyCqpRHqhqeekts5oFQxPXy7DZwmXYHkrlveeWlCOTPNrHkgrghACL2xLQZ/ndyI5pxQAUKCtwb/MYe3j+EzsPlsg9dAcTCvC9pQ8FJbr4OHsiFHmL8fUMgUOppkCaHlNHXak5OGzw5fQMPus2n4OGw5fAgCUVunx6q5UPLU5GUs3J2PFt6cBmMZ6bG0QgD74Kd3qTLi1cel4aMMxGIwCPfyt/6J/amJ3AKbemztf34/SKj2CvVwQ6Ve/Xic3J/i6O6Gy1oAXvz8LAPjLtN7oHaiBUQB3vrEfRzKuorbOiKe+SpYOxzw1qTtGRNbPZzSiWyeM62ka6Pp1oulQ0KQQI+6LCoKlQ3X1fQPw0sz6eZgGhnrBQ+2ICl0dlpqD7qOjI/CHO+pDypxhobh/aKh0/9mpPaVDwnOGheHNOYPg4Wy6vs2DI8Px+uyBVmNDosK8sXJmP+kU4+VTeiLC1w3fPzkKrk7157407PWdOzwMJ1+YjAm9/dHR8eweopvsmak94euuxsOjukpnLNzZyw/v7E1D/2BPfPrQcHioHRGgcUaYjyv6BWugUCgwobc/fjcsFGGdXLFodAQSL13D7rMF0hdPsJcLrpRW49Vdqdh3zvqinzPeiZd+f+F/Z/BxfAZ+zjZtp3JQQG8QeP+n+gnqdHVGrNp5AVlXlLhWVX8YYf+FIqSYLydQrTfgp9RC1BmF1XibhIwSfLg/A//aafoSHNbVByWVOnxrPuYPmLrH//LtKVwoMG2373whfjcsFL//+CgKtTp8/shwHE4vwcYGh6qSsq4hraAcT355Arml1fj2iZGI9HPHV8ez8ZdvT0MI4PilRPz+jjAkmA8H7DlXgAUjuuKzBn9hA6bxAbml1fjkUCaevLM7+gV7WgUqAIhLLcJy83WcjhQqoaszorCyGn/4+CgyiiqhclDg0LPe+CoxB0Zh6nFKL6xA106uWP7NSegNpm/vjUey4Ouuxr92noeH2hFxy8fhHz+ew9k8Lc7mabFwdAT6BXtibVx9/T8/nAVvVyc89OlxGITAE+O64T9Hs3HVfAjCyUGJcl2d1V/88enFOJNbhk/i699rUbkOq3ecx2fmL+J/H8jAew8MxgvbzkiTD5ZV6/HYxiRpG6MwjXsBgEl9/BGgcUZ8ejF2XnYAUJ9I3tqTJs26PHtICLYkXZYOVY7r2RlxqUVWvV8HLhShzmDEvw9koM582YraOiP2mMfWjO7ui0PpxdJ+2cnNCe/OjcKUtw4AABQKYOmE7tiWnIvM4kpU6OoQ6eeO9X8Ygv8czZb2wTfnDMK25CvSYSgXlQN+OyQEZVW10gDXEG8X9A/2xI6UfCgUwAPDwzCimy+uVerxxRFTm/82ozdOXS5DXKopQEWHe2OYbxHCfFzx7z8MhUIBTOjtj6uVtXh7Txo6uTvh0weHYWNCFtbsMR0a6hXggSfGd4OLygFT+vqjrFqPyX394eSgxKNjIuCoVGDR6Po/Plpj8fhILB4f+Yvr3S5nBipEBxj9ptVq4enpibKyMmg0ml/e4FfS6/XYvn07pk+fbtdXuWQdbFODy9eq4K9xhuoGpz829NXxbDz7X9Nfp4+P7YZBoZ54/Iv6s4WGh/ugb5BG6gL3clXBXe0ofbG4qBwwpIs3pvYLkA4JOTkq8a/7+mPZ1yelv9BVDgo8MDwMnyVkYWCIJ1ILyqWem+n9A5BWUIG0wgrMiw6Tei4aCvVxwbVKPSp0dZgzNBTnC8px0hysLKb3D8DISF/87TtTO9ycHKQv0XsGB+OHU7nSl77FA9FhGN/TD49tTIRRAP2DPaXrMVk4OSix46nRuOvdeOkLFDB9IR40n10xKtIX6+cPwYjV+1Ba1fx4lY/nR2HryXx8f9J6rpwvj2ZDW2Ma3/DM1J5QKhRYveM8HJUKqeeooTsifHAko/5QxgPRYbhncDB+t/6INGiyubN+O3uosXBUOHak5EvhNMLXDXqjETlXq63WdVE5SKfCWzgqFXh4VDjWN5iTY6t57h8PZ0csiOmK9xr0Xn28YCiqag148sv6Hp/HxkRg45EsqZ73DA7G87/pg8Evx0ptjH92PNbvz8Cnhy9JwQoAPn94OBZ9nghdnREvz+yLv5sHnALA4b/ciS2Jl7H7bD66+rrhzxO7I9LPA18ey8Zbey7gg3lDMKSLN5KyruJ/ybnoH+KF6f0D4OrkiMLyGmxMyMLMQcGI9HNHbmk1PohLR8oVLe6NCsb8mK4o1NZg2dcn0TdYgz/d2R0uKgcczbyKbp3dpF6JGr0By785hYEhnlg4OgK6OgPe2H0B4b5uuGdgAHbt3NHkZ0KN3gClQgEnRyUKtTX4zbvx6BnggfceiIKny+31GdrWz8a2fn8zpNj5lzPAOgAdowYVujo8tfkEBod544lxpusSfZ6QhdU7zsNZpcT3T47ChYJyaSbd52b0RmcPNZZuToaDUoEvHolGTLdOqK41oPfzOwEAz07thT+O64avj+fgb1tPQ28QeHR0V8yN7orxr8dJr+2hdpTmhwEAjbMj4v9yJya8sV8aYzKhlx/2nq8/A+WOCB98/nA0vv35Mv7y7Wk4q5QY2sUH8ebBvt6uKlyr0kPtaOq1cFQq8MS4bvjzpB6Y9cHhRsGmod8NC8Wqe/tjzZ406dCT5cKRFkO6eCM63AcfNOixsPjDHV2w8UgWunRyxd0Dg/Duvvov6j6BHjibV46YCB8cybwKIYAnxnVr8nkASD0Er943AO/9lC6NU7hncLDVOI/h4T44lnkVakclFArToOjJffyhVjlIQWhsj84wGAXi04vx4Iiu+NuM3lA5KPHc1tPSX/w//mkUfjyVhw/iLsJRqYBSoUBnDzXmDg/F6+b5Nqb3D8CVa9U4ebk+xL1wVx/cMzgYv//4KLxcnPDPe/rBX+OMv31nOqW2SydXvP27wci+WoWJb+4HAPQO8MC3T4xEzrUq/PfnyyivqcOKab3g4azCzPfikZKrxX8WRuMO82UgjEaBito6LP7PzziYVgwPZ0eU19RhYKgXtj4xAuErtgMA5kWH4Z/39G/231cOWvKZYDSKZud16ehsHVJ4uIeog3BXO+KjBcOsli0Y0RX3RAXDaBTwcnWCt6sTAj2d4emiwu/v6AInByVKKmrRpZMrYrqZvkhcnBzwxuyBuHytGo+OMXU93z8sFD38XLFxxyH86c5IuDk7oWsnV+lyACum98ZH8RnSYMbHxnaDxlmFedFheHtvGp68szv+PLE75n9yDAfTijG1bwDeuH8gnByVmDMsFCO6+cJPo4ajUoGef98Jg1HgWpUeEZ3d8NlDw7H/QhEm9fGHv/kv3N4BHlJIObB8PB7ccEx67fE9O+PlWf2gUCjw54ndoXF2RHWtAYXlOulwkYNSgZdn9sPVylopXPx2SAiuVtZi3/lCab1Hx0QgQFM/983QLt54bExXLNp4Agnm3o+Jvf3x50k9sC05VxrTM3d4GDYfz4YQprEj43p2xuyhIQjv7IbNx3Lw8Kiu6B2gwdlcLVILynHv4GC8+tsBmPFOPFLNZxKNM59pUVtnxJ29OqNbZ3f0C/KEAJCvrUFwg9mIHxoZjqySKiwcHYG+QZ6I9HNHVJg3BoV5wdNFBQeFApW1dcgsrkJ0uA9mDw3BD6fy8OSXJ+CicsDDo7riwRFdoVAo8MOTo632oTfuH2h1P9zXDZGd3XBVW4F1vx8MFycH9PD3wIppva3W+/Sh4SitqrUaP6FUKqBxVmFwmDcOphWj3NzjtGR8JBQKBT55cCji00rwf1N6NLmPd1S3a0CRA4YUog5O41z/142b2hH7l4+HgIDa0XS20cOjwhttc5/5bIOG+gZpMCZQmP/SV2DNnEHYf6EI43v6YWCoFyb09sPJnFJ4uzlhiHng79IJ3fHIqHBp4N+Hvx+C1IJyDArxkj64FQqF1RkEIyN9ccA8cPKDeVEI9XHF7xsMNARME+WdvFyGx8dGIKyTK5ZO6I4/f5WMedFd8PxdfaRDZAqFQjqj4WROKb46noMunVyxYnov9AnSoLbOiPE9O8PXXY1X7u2PwxdLsM/c2zOuZ2fcFxUifZECwB9iuiAmohP8nAUKaxRwc3LAskk9oHJQYsNDw5CQUYIQbxeM7t4Z7moHHEwrxpAu3nh6ck8oFAoM6+qDYV19pOf75KFhOH25FJP6BMBBqcDGhcNxIrsUGmcVhof7wEGpgIuTA+4ZbP3vEXzd5RK6dXbHxkeipftqRwfpLA0LD2eVVeC4a2AQegZ4INDTWfr3+TUclAp8vzgG23fsRGCDyQuv5+Pm1OzVv6PCvKTfh4f7YGJvPwDAnb38pYtxEv0aDClEtxknx/Y5aW9wmLfVWUj+GmdM7htgtY5CobD6AnRTOyKqwTZNeWpidzgogD9N6G51SmVD3f09sGNp/V/8MwcFY2q/ACl4NWVgqBdSVk6BykEhDRp0clTi04eGS+uM7dEZ6/8wBBoXFaLDfaBQmGYXXhDTBcWVtaaJ+4wGrBhkwOg7J0Hj6izVs7u/B7r7e0jP9bcZfW74PgFT2GgYOPw8nDHluhreTD0atLclHB2UaMtuNDi0fh/4+4w+t80gTrr1GFKI6JaKCvO2Cg6/1o0CisWvCWjXBy0AWDmzn/S73miAUgF4uqigaqfAZ288XVXY+MhwGIwC/UM8bd0c6sAYUoiIqN2N7i6vq+lSx8Q/E4iIiEiWGFKIiIhIlhhSiIiISJYYUoiIiEiWGFKIiIhIlhhSiIiISJYYUoiIiEiWGFKIiIhIlhhSiIiISJYYUoiIiEiWGFKIiIhIlhhSiIiISJYYUoiIiEiWOsRVkIUQAACtVtuuz6vX61FVVQWtVguVStWuz92RsA6sAcAaWLAOrAHAGli0tQ6W723L93hLdYiQUl5eDgAIDQ21cUuIiIiopcrLy+Hp6dni7RSitfHmFjIajcjNzYWHhwcUCkW7Pa9Wq0VoaChycnKg0Wja7Xk7GtaBNQBYAwvWgTUAWAOLttZBCIHy8nIEBQVBqWz5CJMO0ZOiVCoREhJy055fo9HY9U5owTqwBgBrYME6sAYAa2DRljq0pgfFggNniYiISJYYUoiIiEiW7DqkqNVqvPDCC1Cr1bZuik2xDqwBwBpYsA6sAcAaWNi6Dh1i4CwRERHZH7vuSSEiIiL5YkghIiIiWWJIISIiIlliSCEiIiJZsuuQ8v7776Nr165wdnZGdHQ0jh07ZusmtcqqVaswbNgweHh4wM/PD7NmzUJqaqrVOuPGjYNCobD6efzxx63Wyc7OxowZM+Dq6go/Pz8sX74cdXV1VuvExcUhKioKarUakZGR2LBhw81+e7/aiy++2Og99urVS3q8pqYGixcvRqdOneDu7o777rsPBQUFVs/R0WvQtWvXRjVQKBRYvHgxgNtzPzhw4ADuuusuBAUFQaFQYOvWrVaPCyHw/PPPIzAwEC4uLpg4cSLS0tKs1rl69SrmzZsHjUYDLy8vPPLII6ioqLBa59SpUxg9ejScnZ0RGhqKV199tVFbtmzZgl69esHZ2Rn9+/fH9u3b2/39NudGddDr9Xj22WfRv39/uLm5ISgoCPPnz0dubq7VczS1/6xevdpqHTnX4Zf2hQcffLDR+5s6darVOh19X/ilGjT1+aBQKPDaa69J68hqPxB2avPmzcLJyUl88skn4syZM2LRokXCy8tLFBQU2LppLTZlyhTx6aefipSUFJGcnCymT58uwsLCREVFhbTO2LFjxaJFi0ReXp70U1ZWJj1eV1cn+vXrJyZOnChOnDghtm/fLnx9fcWKFSukdTIyMoSrq6tYtmyZOHv2rHj33XeFg4OD2Llz5y19v8154YUXRN++fa3eY1FRkfT4448/LkJDQ8XevXtFYmKiuOOOO8SIESOkx2+HGhQWFlq9/9jYWAFA/PTTT0KI23M/2L59u/jb3/4mvv32WwFAfPfdd1aPr169Wnh6eoqtW7eKkydPirvvvluEh4eL6upqaZ2pU6eKgQMHiiNHjoiDBw+KyMhIMXfuXOnxsrIy4e/vL+bNmydSUlLEl19+KVxcXMS6deukdQ4dOiQcHBzEq6++Ks6ePSuee+45oVKpxOnTp296DYS4cR1KS0vFxIkTxVdffSXOnz8vEhISxPDhw8WQIUOsnqNLly7ipZdesto/Gn6OyL0Ov7QvLFiwQEydOtXq/V29etVqnY6+L/xSDRq+97y8PPHJJ58IhUIhLl68KK0jp/3AbkPK8OHDxeLFi6X7BoNBBAUFiVWrVtmwVe2jsLBQABD79++Xlo0dO1YsXbq02W22b98ulEqlyM/Pl5atXbtWaDQaodPphBBCPPPMM6Jv375W282ZM0dMmTKlfd9AK73wwgti4MCBTT5WWloqVCqV2LJli7Ts3LlzAoBISEgQQtweNbje0qVLRbdu3YTRaBRC3P77wfUfykajUQQEBIjXXntNWlZaWirUarX48ssvhRBCnD17VgAQx48fl9bZsWOHUCgU4sqVK0IIIT744APh7e0t1UAIIZ599lnRs2dP6f79998vZsyYYdWe6Oho8dhjj7Xre/w1mvpyut6xY8cEAJGVlSUt69Kli1izZk2z23SkOjQXUmbOnNnsNrfbvvBr9oOZM2eKO++802qZnPYDuzzcU1tbi6SkJEycOFFaplQqMXHiRCQkJNiwZe2jrKwMAODj42O1/D//+Q98fX3Rr18/rFixAlVVVdJjCQkJ6N+/P/z9/aVlU6ZMgVarxZkzZ6R1GtbMso6capaWloagoCBERERg3rx5yM7OBgAkJSVBr9dbtb9Xr14ICwuT2n+71MCitrYWX3zxBR5++GGrC3Paw35gkZmZifz8fKv2enp6Ijo62urf3cvLC0OHDpXWmThxIpRKJY4ePSqtM2bMGDg5OUnrTJkyBampqbh27Zq0TkepC2D6nFAoFPDy8rJavnr1anTq1AmDBw/Ga6+9ZnWo73aoQ1xcHPz8/NCzZ0/88Y9/RElJifSYve0LBQUF+PHHH/HII480ekwu+0GHuMBgeysuLobBYLD6IAYAf39/nD9/3katah9GoxFPPfUURo4ciX79+knLH3jgAXTp0gVBQUE4deoUnn32WaSmpuLbb78FAOTn5zdZD8tjN1pHq9WiuroaLi4uN/Ot/aLo6Ghs2LABPXv2RF5eHlauXInRo0cjJSUF+fn5cHJyavSB7O/v/4vvz/LYjdaRSw0a2rp1K0pLS/Hggw9Ky+xhP2jI0uam2tvw/fj5+Vk97ujoCB8fH6t1wsPDGz2H5TFvb+9m62J5DjmpqanBs88+i7lz51pdNO5Pf/oToqKi4OPjg8OHD2PFihXIy8vDm2++CaDj12Hq1Km49957ER4ejosXL+Kvf/0rpk2bhoSEBDg4ONjdvvDZZ5/Bw8MD9957r9VyOe0HdhlSbmeLFy9GSkoK4uPjrZY/+uij0u/9+/dHYGAgJkyYgIsXL6Jbt263upk3xbRp06TfBwwYgOjoaHTp0gVff/21rL44b5WPP/4Y06ZNQ1BQkLTMHvYDujG9Xo/7778fQgisXbvW6rFly5ZJvw8YMABOTk547LHHsGrVqttievjf/e530u/9+/fHgAED0K1bN8TFxWHChAk2bJltfPLJJ5g3bx6cnZ2tlstpP7DLwz2+vr5wcHBodGZHQUEBAgICbNSqtluyZAl++OEH/PTTTwgJCbnhutHR0QCA9PR0AEBAQECT9bA8dqN1NBqNLEOAl5cXevTogfT0dAQEBKC2thalpaVW6zT8N7+dapCVlYU9e/Zg4cKFN1zvdt8PLG2+0f/1gIAAFBYWWj1eV1eHq1evtsu+IafPFEtAycrKQmxsrFUvSlOio6NRV1eHS5cuAbh96mAREREBX19fq/3fXvaFgwcPIjU19Rc/IwDb7gd2GVKcnJwwZMgQ7N27V1pmNBqxd+9exMTE2LBlrSOEwJIlS/Ddd99h3759jbrhmpKcnAwACAwMBADExMTg9OnTVv9BLR9iffr0kdZpWDPLOnKtWUVFBS5evIjAwEAMGTIEKpXKqv2pqanIzs6W2n871eDTTz+Fn58fZsyYccP1bvf9IDw8HAEBAVbt1Wq1OHr0qNW/e2lpKZKSkqR19u3bB6PRKIW4mJgYHDhwAHq9XlonNjYWPXv2hLe3t7SOnOtiCShpaWnYs2cPOnXq9IvbJCcnQ6lUSodAboc6NHT58mWUlJRY7f/2sC8App7WIUOGYODAgb+4rk33gxYNs72NbN68WajVarFhwwZx9uxZ8eijjwovLy+rsxo6ij/+8Y/C09NTxMXFWZ0yVlVVJYQQIj09Xbz00ksiMTFRZGZmim3btomIiAgxZswY6Tksp55OnjxZJCcni507d4rOnTs3eerp8uXLxblz58T7778vq9Nvn376aREXFycyMzPFoUOHxMSJE4Wvr68oLCwUQphOQQ4LCxP79u0TiYmJIiYmRsTExEjb3w41EMJ0plpYWJh49tlnrZbfrvtBeXm5OHHihDhx4oQAIN58801x4sQJ6ayV1atXCy8vL7Ft2zZx6tQpMXPmzCZPQR48eLA4evSoiI+PF927d7c67bS0tFT4+/uLP/zhDyIlJUVs3rxZuLq6Njrl0tHRUbz++uvi3Llz4oUXXrilpyDfqA61tbXi7rvvFiEhISI5Odnqc8Jyhsbhw4fFmjVrRHJysrh48aL44osvROfOncX8+fM7TB1uVIPy8nLxf//3fyIhIUFkZmaKPXv2iKioKNG9e3dRU1MjPUdH3xd+6f+DEKZTiF1dXcXatWsbbS+3/cBuQ4oQQrz77rsiLCxMODk5ieHDh4sjR47YukmtAqDJn08//VQIIUR2drYYM2aM8PHxEWq1WkRGRorly5dbzY8hhBCXLl0S06ZNEy4uLsLX11c8/fTTQq/XW63z008/iUGDBgknJycREREhvYYczJkzRwQGBgonJycRHBws5syZI9LT06XHq6urxRNPPCG8vb2Fq6uruOeee0ReXp7Vc3T0GgghxK5duwQAkZqaarX8dt0Pfvrppyb3/wULFgghTKch//3vfxf+/v5CrVaLCRMmNKpNSUmJmDt3rnB3dxcajUY89NBDory83GqdkydPilGjRgm1Wi2Cg4PF6tWrG7Xl66+/Fj169BBOTk6ib9++4scff7xp7/t6N6pDZmZms58Tljl0kpKSRHR0tPD09BTOzs6id+/e4pVXXrH6AhdC3nW4UQ2qqqrE5MmTRefOnYVKpRJdunQRixYtavSHaUffF37p/4MQQqxbt064uLiI0tLSRtvLbT9QCCFEy/peiIiIiG4+uxyTQkRERPLHkEJERESyxJBCREREssSQQkRERLLEkEJERESyxJBCREREssSQQkRERLLEkEJERESyxJBCREREssSQQkRERLLEkEJERESyxJBCREREsvT/+7jP5Uhd/moAAAAASUVORK5CYII="
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 推理"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "def generate_text(model, start_string, max_len=1000, temperature=1.0, stream=True):\n",
    "    input_eval = torch.Tensor([char2idx[char] for char in start_string]).to(dtype=torch.int64, device=device).reshape(1, -1)\n",
    "    hidden = None\n",
    "    text_generated = []\n",
    "    model.eval()\n",
    "    pbar = tqdm(range(max_len))\n",
    "    print(start_string, end=\"\")\n",
    "    with torch.no_grad():\n",
    "        for i in pbar:\n",
    "            logits, hidden = model(input_eval, hidden=hidden)\n",
    "            # 温度采样\n",
    "            logits = logits[0, -1, :] / temperature\n",
    "            # using multinomial to sampling\n",
    "            probs = F.softmax(logits, dim=-1)\n",
    "            idx = torch.multinomial(probs, 1).item()\n",
    "            input_eval = torch.Tensor([idx]).to(dtype=torch.int64, device=device).reshape(1, -1)\n",
    "            text_generated.append(idx)\n",
    "            if stream:\n",
    "                print(idx2char[idx], end=\"\", flush=True)\n",
    "    return \"\".join([idx2char[i] for i in text_generated])\n",
    "\n",
    "\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)\n",
    "# load checkpoints\n",
    "model.load_state_dict(torch.load(\"checkpoints/text_generation_lstm/best.ckpt\", map_location=\"cpu\"))\n",
    "start_string = \"All: \"\n",
    "res = generate_text(model, start_string, max_len=1000, temperature=0.5, stream=True)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-12-13T08:14:22.160236Z",
     "iopub.execute_input": "2023-12-13T08:14:22.160992Z",
     "iopub.status.idle": "2023-12-13T08:14:24.003550Z",
     "shell.execute_reply.started": "2023-12-13T08:14:22.160952Z",
     "shell.execute_reply": "2023-12-13T08:14:24.002573Z"
    },
    "trusted": true
   },
   "execution_count": 16,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "  0%|          | 0/1000 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "139b7c6d40a747a6ac66361dcc3945e3"
      }
     },
     "metadata": {}
    },
    {
     "name": "stdout",
     "text": "All: thou art no cause to stay.\n\nHORTENSIO:\nSee what I was worse, if it were so; list,\nBecause so well I love Lucentio.\n\nLUCENTIO:\nAnd will be sworn yet on my father's steed,\nAnd not been calm, bid heaven both to hard!\nO man, bethinks, as yourself and men,\nScarity and paper hence he should have drawn her.\n\nLUCIO:\nSome say he is, his stoutness\nDespites me are full of a charm to hear\nMy son and metry days ago.\n\nMIRANDA:\nWhy dost thou speak that we do?\n\nCLARENCE:\n\nBUCKINGHAM:\nMadam, him that slanders here? the old mischance\nWill yield to all ill hope in these warses' mouths.\nI come, we know your meaning, brother\nA precenent full of highest crown.\nIf thou hadst fear'd to break an oath by Him,\nThe imperiage was a famour life\nThat our renowned household hands too;\nFor, ere thou not, Jule, and grant of prin.\n\nYORK:\nWhat, doth your lordship proportion of your young prince\nExcepting Camillo was Antigonued.\nWhich often have you both our tongues to get him for his lack?\n\nPay:\nHow fares my Juliet? how ",
     "output_type": "stream"
    }
   ]
  }
 ]
}
